{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bc6ddf0e-2247-4cd2-927d-6ad825704e05",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "220.00000000000003\n"
     ]
    }
   ],
   "source": [
    "# 该书中绝大多数基础知识都和高等数学、线性代数相关\n",
    "\n",
    "# 误差反向传播法(基于数学式/基于计算图):局部计算(特征)\n",
    "# 计算图:1.构建计算图;2.在计算图上,从左向右进行计算\n",
    "# \"从左向右计算\"是正方向上的传播,简称为'正向传播'(从计算图出发点到结束点的传播)\n",
    "# \"从右向左计算\"则被称为'反向传播'(重点)\n",
    "# 优点:1.局部计算;2.可以将中间节点全部存储起来\n",
    "# 苹果的价格上涨会在多大程度上影响最终的支付金额 = 支付金额关于苹果的导数 = ∂L/∂x(苹果价格为x, 支付金额为L)\n",
    "# 反向传播传递\"局部导数\"\n",
    "\n",
    "# 链式法则\n",
    "# 将信号乘以节点的局部导数(∂y/∂x),然后将结果传递给下一个节点\n",
    "# 链式法则是关于复合函数的导数的性质:如果某个函数由复合函数表示,则该复合函数的导数可以用构成复合函数的各个函数的导数的乘积表示\n",
    "\n",
    "# 反向传播:一般把构建神经网络的\"层\"实现为一个类,本书中的\"层\"是神经网络中功能的单位\n",
    "# 如:乘法节点称为\"乘法层\",加法节点称为\"加法层\"\n",
    "\n",
    "# 乘法层的实现\n",
    "# forward()对应正向传播,backward()对应反向传播\n",
    "class MulLayer:\n",
    "    # 初始化:创建实例初始化两个属性,self.x/y属于实例变量(每个MulLayer实例有独立的x和y)\n",
    "    def __init__(self):\n",
    "        self.x = None\n",
    "        self.y = None\n",
    "\n",
    "    def forward(self, x, y):\n",
    "        self.x = x # 存储输入x到实例\n",
    "        self.y = y # 存储输入y到实例\n",
    "        out = x * y\n",
    "\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout): # dout上层输入\n",
    "        dx = dout * self.y # 翻转x和y,使用前向传递保存的y\n",
    "        dy = dout * self.x # 使用前向传递保存的x\n",
    "\n",
    "        return dx, dy\n",
    "\n",
    "# 正向传播实现\n",
    "apple = 100\n",
    "apple_num = 2\n",
    "tax = 1.1\n",
    "\n",
    "# layer\n",
    "mul_apple_layer = MulLayer()\n",
    "mul_tax_layer = MulLayer()\n",
    "\n",
    "# forward\n",
    "apple_price = mul_apple_layer.forward(apple, apple_num)\n",
    "price = mul_tax_layer.forward(apple_price, tax)\n",
    "\n",
    "print(price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "76fa3fc7-9f8d-40fc-915b-407c49f2b879",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2 110.00000000000001 200\n"
     ]
    }
   ],
   "source": [
    "# backward\n",
    "# backward()的参数中需要输入\"关于正向传播时的输出变量的参数\",mul_apple_layer乘法层在正向传播时会输出apple_price,\n",
    "# 反向传播时则将apple_price的导数dapple_price设为参数\n",
    "dprice = 1\n",
    "dapple_price, dtax = mul_tax_layer.backward(dprice)\n",
    "dapple, dapple_num = mul_apple_layer.backward(dapple_price)\n",
    "\n",
    "print(dapple, dapple_num, dtax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1ea0d0bf-ba33-4354-b276-d624244734d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加法层的实现\n",
    "class AddLayer:\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    def forward(self, x, y):\n",
    "        out = x + y\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        dx = dout * 1\n",
    "        dy = dout * 1\n",
    "        return dx, dy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "76e67a98-bc9b-4d1b-9988-ac0f0fbd5179",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "715.0000000000001\n"
     ]
    }
   ],
   "source": [
    "# 使用加法层和乘法层实现例子\n",
    "apple = 100\n",
    "apple_num = 2\n",
    "orange = 150\n",
    "orange_num = 3\n",
    "tax = 1.1\n",
    "\n",
    "# layer\n",
    "mul_apple_layer = MulLayer()\n",
    "mul_orange_layer = MulLayer()\n",
    "add_apple_orange_layer = AddLayer()\n",
    "mul_tax_layer = MulLayer()\n",
    "\n",
    "# forward\n",
    "apple_price = mul_apple_layer.forward(apple, apple_num) # 1\n",
    "orange_price = mul_orange_layer.forward(orange, orange_num) # 2\n",
    "all_price = add_apple_orange_layer.forward(apple_price, orange_price) # 3\n",
    "price = mul_tax_layer.forward(all_price, tax) # 4\n",
    "\n",
    "# backward\n",
    "dprice = 1\n",
    "dall_price, dtax = mul_tax_layer.backward(dprice) # 4\n",
    "dapple_price, dorange_price = add_apple_orange_layer.backward(dall_price) # 3\n",
    "dorange, dorange_num = mul_orange_layer.backward(dorange_price) # 2\n",
    "dapple, dapple_num = mul_apple_layer.backward(dapple_price) # 1\n",
    "\n",
    "print(price) # 715"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ba3f10b0-72f9-4f0d-b666-0cee15140de7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "110.00000000000001 2.2 3.3000000000000003 165.0 650\n"
     ]
    }
   ],
   "source": [
    "print(dapple_num, dapple, dorange, dorange_num, dtax) # 110 2.2 3.3 165 650"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1d92232a-add7-4304-814e-ee96755b7817",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.  -0.5]\n",
      " [-2.   3. ]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "# 激活函数的ReLU层\n",
    "class Relu:\n",
    "    def __init__(self):\n",
    "        self.mask = None # 是由True/False构成的NumPy数组,它会把正向传播时的输入x的元素中小于等于0的地方保存为True\n",
    "\n",
    "    def forward(self, x):\n",
    "        self.mask = (x <= 0)\n",
    "        out = x.copy()\n",
    "        out[self.mask] = 0\n",
    "\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        dout[self.mask] = 0\n",
    "        dx = dout\n",
    "\n",
    "        return dx\n",
    "\n",
    "x = np.array([[1.0, -0.5], [-2.0,  3.0]])\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "151078e9-941e-4d9b-a2cd-62b8dd85c232",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[False  True]\n",
      " [ True False]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAACeCAYAAACvvAnjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2CklEQVR4nO3dd5Bl133g9++59+XcOefpnp6MmQEGwABDgARICCRAiQDFIImSltTKdtnWLl3rlSVvlatc3tJqveva8pZXNteWtyRRJUoWRTEIJBckCJAIgzCYnDrnnF5O9x7/8bp7ZjCpJ/V73e/3qQIKmH7vze/1vffc3z3nd85RWmuNEEIIIcqWUewAhBBCCFFckgwIIYQQZU6SASGEEKLMSTIghBBClDlJBoQQQogyJ8mAEEIIUeYkGRBCCCHKnCQDQgghRJlzbPSF3/3hdzh67OiDjEUIIYQQ91m1v/a2r9lwMpBOZ/jOX/09lmXdU1BCCCGEePCcTgfPffY58N/+tRtOBrKZLFPjk7zw+RcwTBldEEIIIUqVbdl87//7PrlsbkOv33AyABAMBWlsacQ0zbsKTgghhBAPnmVZBIKBDb9eHvGFEEKIMifJgBBCCFHmJBkQQgghypwkA0IIIUSZk2RACCGEKHOSDAghhBBlTpIBIYQQosxJMiCEEEKUOUkGhBBCiDInyYAQQghR5u5oOWJx59KpNKlkqthhbBvBUBCHU05bsTXksjnisXixwyhLSimC4aAsn79B0qo+YK985xW+85ffoamtGaVUscPZ0sZHxvjDP/pDdu7ZWexQhNiQsyfP8sf/4o9p62rH4ZDmdvNopsan+Ff/4V9R23D77XuFJAMPXC6b48ixI3zt934Hw5Bk4F780R/8EbZlFzsMITbMsixq62v5g3/5P+AL+IodTtmwbZs/+sM/wtbSXmyUJAObwDBNXE6nbP18j5Qhvz+x9ShD4XQ5cblcxQ6lqLTWAJvSQ2pZFkpJe3En5Lcl7i+t0av/CCHK00fbAG3bXHrnTU5dmMSWtqEkSTIg7t7ajf+aP8zy7g9/xOm+GeSaF2LruVEyr7XGti0yqTSZbP6GP1+YmiaeymLlskwMDBFN5q56nc3Ae7/gwsjKJn0LcadkmEBcQ2uNnc+RSmUwnW7cHifGVd162s4xOTJFqLYWl2kzfGmIuu5uwj4XCkBbXDj+Hs21u4v2HYQQd05rm3R0mb7LE7Tv6yXkca7/LJ+K8pPvvELOV0VqboqGQ8c4eqgdc61t0Ba/+Ju/ItWyn/awyU+//X32ffm/4oWjHeufoQyTUDhILpPF6XZd066I4pNkQFyhNanFSb7/7Z8SaWlmfmCQtqOf5PGHWq5cuPk0r/7Vt6g6eIxaf5rv/dn3+dR/+9/x5P7GK59jOgj43SQTSbx+r1z0QpQwbVvEl+Y58+4JhiZjdB04iNthXv0CTv3w23w4U8nXf+8ZUqOn+Tf/9i9oav1ndNR4Wbu6c5bJnkcPs7fK5OJbb1HlVZw5cWb9MyZmlsif+4Bv/HSMZ37zN9jVEpEZViVEkoHtTFtMjYzhitRQEfbd9qas7Ryv/81fsxh6hJc+9QSLFz382z/5S9r/56/TVOFeexHadLPnyMO0BaO8+cp7hIly8vg0GgU6w/TcMur4z3nzr6f5ld/9Lbrqg5vwZYUQWmusbIZkKovb5wMrSzan8QV8mB+ZzaTtPMsz05w8foLJpRy9hw/yuY+34HU7rrlJ2+llXvvR+7R/4fdwmwaO+k6q1Sxvf9BP+3N7119r5fPkczmyWRvL1niCEdpqqgo/ty36KwLojl4+/rmn8AalTSg1kgxsVVqTTsaJJ3MEIyHyyQQZSxGOBK+56HPxRX7+07cJ1Ldy8JF91FUHb5oUZONzvPXWJZ76J1/CVIrqtg68K/+J985P0vhEx+owgMbK58mmM2ScGSxbE6yuozbsLlz0OsWJihAdDx3h5ZeqcHu9m/P7EEKQjS9y6sN+HCS50L9Ia0cjfe+/x6HPfomHdhbm22utWZke5e3X32Mp62DfkcMc7WrE5TBu+KSeXllidDrOgWo/CjAdLqoibvovjqI/tZfVhoFcJkX/6XPkggbzy0lMp4Ol6UlCzR1UBA08LifaH6CiugLpDyg9UkC4FWlNfH6CD09cYvjsCb75p9/m9MmzfPPffYNz41cV6CiTlj0HefkrL7GrxcdbP/gB3/v7nzE6tYR1g+q+zPwk08t5QuEAKFAePxUBxcDlsSvFgBqyyTh9Z85y5uRFlhNZDFMx3j9ETpt4vR4cDgO3x0sg6MfpkFNMiM2hGbs8QP2ufezZ08L5d89Q39PDzr07qa70X/O6mYFLXBxaoHP3Lro6GnA5zJt22eeyWTI5C8fatawUTpeDZCxxTZGwVk52HjzA4UcfoqbCi8bETE7xF//v90hkZb5/qZOWegvS2mJ4YIqeg/vp7Kim78wAHfv3cPjYYzRWXPskrpTCdLpp3/sQn/uNz3N4Vy0fvvpj/u5vfszA6Pw103ysVJqsVpimUcjclYHLaZJKJLHX/26NcvnY/cghHn5sP5V+FxgOzOgwf/YXr5KxZAqBEMXSsnsfTdVelsYncTa00VBTwdFf+hRN1VcveKToeeKT/O5/+RJqaYTvfPPbvHX8HLFU7oZTgp0uF26nST6/2gpoTS6TJxgJcXX+YFk2ylAYhlrvRWzo6iYx3sdiNPsgv7a4DyQZ2IqUSc/BA1T6nSwODeDv2ENNJMzjzz5BVcB947cohWE6aOrZxTPPH8NcGuT7f/868eyVi98R8OMxNPm8VZguqPOkM3kiFRHWy4m0Llz0KAr9gxqNoqG3l8TwAPG8PAEIUQxaF27cSmuGL12krrMbrxOWZufI2lff5DWDJ45zum+evcc+zuc+/0l8mTm+/5d/y8/eOMlyLH1NUuCJVNLeGGBuNo7WYOUzzC1l6N3dwdqIpNY2Vi7LwvQMUxMzJFI5bA1mqI7f+frXaKi4cbskSockA1uUUgpt57hwZoiufTsw0MxNTJG9wZO51hptW0Tnpnjje//Ad7/3Fi1HPsnv/OPPEnRdSe3d1c201LhZWFgu3PRjMeaSDnbvbQV15bPy2TTTY+OMjUySSOfQGryVbfwX//1vUumWTUGEKAYrscif/vG/51TfKOfPjlLTVE92ZYFzF0Y/8kpF444enPFJvvvNv+Xd02N0Pfw4n/vSp6n3pnntH95gKZVbf7XhDvPcL3+MsdNniWdyzA9dIObr5PHD7VzpGrCx9WqPgAKn14fLqUA5qKmvwWFKlUCpkwLCLSi7PM6f/Jtv8Ymv/AoDY8sceCJAdGaUvpE4RxobrnmtbeVZnp7gxDsnmE8odj9yiCO/1IjH6eCjQ4ROXyXPvXCUd06fI3mglvEzp/F1P8LBntr1gh+tNSgTj9eD15snXBHG6TAwTAcVlWGwk5vzSxBCXMNweujZ201seo7HP/9FpiYmOXMhxq6De3FdVVSslMIbruTwxz/B3kei9J05wz9869tUte3g4OH9dD3kxOG4ssYAymD3M7+MUXGKy2cukI3l+a1/+ts0hl3r7YJSJkc+/RxtuzoIuw2+9F//Lr6wF7QmOjfFuVPnOXN5ir0Pb+7vRGycJANbkMMX5uEnD5JeTvDiV3+dyckxBnQF+w/txHn19CGd48zrr3FpJsf+I49xrLUWl9O8aSWvMkwOffolAmcuMHCpnzQ1/O5/8zQRz5XTxHD5eebzv0xnRx1eRw1f/fpX8QY8aG2zMDbMqQ8+5NzQPLtl/rAQm8pw+zn2wvPYGgxDsXOPjVYKwzBueM0rpfAEw+x9/Al2HkwycuE8P/vO9wg0dnHs6UP4XVd6+Qynh11HjhTGIpS6rthQGS4OHH1k/f/DlRGg8PDgq6iiLmRiV3TS217xIL66uA8kGdiCTFeQJ577OBqFUtCxs3CBXj9l0MGuo0+xx+m8UhR4SwrT5aX30MErf/KRzzTcXvYe2rsWCYFQoUpZa6hobKZlYoDmvYfpaAjdy1cUQtwFZRjr9T3K3NiQnVIKl9fPjoMP07F3H7OTcyiuH25USnFdd+IGPtvhdNPxyFP8s0PHcDhu/jAiikuSga1oNTO/0kV3k8tLKVyeOy/cuZtVwZRSmA4X3Y89Q9cRZLtmIbYYpRQOl4fG9pb7/9mGgVN2HS1pkgxsovLYyU+xds2Xx/cVYmPkehClTJKBTTA1Nslbr7+FIZnxPVmYmy92CELcsdhKjOO/OI7H4yl2KGXDtm2WF5eKHcaWIsnAA9bW2cZQ3xBv/vTNYoey5TW2NBKKSC2C2DqqqqvYuWcn7/3ivWKHUnZa2lvwynLoGybJwAP26LFHefTYo8UOQwhRBB3dHfz+//L7xQ5DiNuSfmshhBCizEkyIIQQQpQ5SQaEEEKIMifJgBBCCFHmJBkQQgghypwkA0IIIUSZk2RACCGEKHOSDAghhBBlTpIBIYQQosxJMiCEEEKUOUkGhBBCiDInyYAQQghR5iQZEEIIIcqcJANCCCFEmZNkQAghhChzkgwIIYQQZU6SASGEEKLMSTIghBBClDlJBoQQQogyJ8mAEEIIUeYcxQ6gVGit0baNbWsMh4m2LDQK0zRQShU7vLumtQbAtjW5XI5MOkM+l8eybQAMw8DpdOB2u3C6XBhG4btu9e+sbRtbg2kaa3+IVgpjC38vIe6ntbZBa00ulyeTzpDL5bAsGzQYpsLpdOJyuXC5nRhG4Vra8m2DLrTzpmmilEJrG60VSim28Fe7Z5IMUDhB0ivznDs7SGxxDttfg1+luXBpkhd/7SWqw+5ih3hHtNagIZPNsrIUZXlphUw6g+EwcTmdOJyO9ZukZdnkc/nVRsDC5XYRqQgTqQjjcrlQaqtd/Jp8coXTH5xjaHiao599kYawyfuvvIJnz1H2tVcWO0CxRWltszw9yfjkIhWNzTgyy8wspOjY1UPI5yx2eBuylgDksjlWVmIsLy6TSqULDwUuJ06HA9NhAgrbtsnlcuSyeSwrj9PpJBwJEakM4/G4V2+eW6ltAJ3PcOHEGYb6h2g/+kn2tEcYeu9NxowmPna4A9ha3+d+kmQAQFtcOttHy96DMPEu/9s33ub3/vnnGRyY2lKnRiHr1cSicWam50inMoTCAeoba/H6vJimgWFc29Nxdc+BZVmkU2kWF5a5PD2Ax+Oitr6WUDiw/lRQ6rTWjF3uJ9TcwuyPXmV2+VPUe7K88dr7PLv3yWKHJ7YorTXLE0NcmszQXuviz//9Nzj64qe58KPvMW/+Fs881FzsEG9L25pEIsnM9CyJeJJAMEB1bRV+vw/TYV7XNsCVNsXKW6QzGZYWVxjsG8Z0mNTV1xCOhG74vlK1OD5MLlCDNfsTBidW2NPi5f033sR/9IvFDq3oJBkAUAYde/cSDHu48NYkVb27qG9o5Itf/dUtdRNMJJJMjE5hWRZ1DbWEI6HbDnOs/cw0C0MiLpeTYCiAZdlEV6JMTkwxPWnQ1NJAIOjfAhe9oqazm+TkBWbsOjoafMSmR4iqEE3VwWIHJ7awWCJH755uHLF+YjHo3NvDjpZ/RLCmttih3ZLWmnQqzcTYFKlUmrqGWlrbW3A4zNtez2tP/4ar0HMQCPixm+uJRRNMT80yNTFDU0sj4UhwC7QN4K9ppDOxyA/GbL786zXkklFGp1K80FlPWY8RIMkAAEoZhCMhtJXh4sVReh5+GENBPJbEFwpiFjvA27Asi6mJGRYXlmlsrqeyKnJPXXhKKRwOk4rKCJGKMMtLKwwPjhKpCNPQVI/DUbq/EaUUgVCAcz86T8O+vQRcBv0DF/A2dhPxbY3ETpSmlp6dAIycHMTRsoMqnxtnoKHIUd2abdvMzswzMzVLfWMdHTva7ulJXimFaZqEI0FC4QCxaJyxkQkWF3y0tDbidJX2cInHH2Tu4nF0VRv1lT7iQ2dZcTbSVOnaUr3AD4K0jmjS82P82f/510xOT9E3vEhzUz3pxSlOnh5GFzu8W9Bak8lkuXxhgHQ6Q++ebqqqK+5bt51SCsMwqKiM0Lunm1wuT9/FATKZzPrwQknSmpVogtraarDSnDp+kY79PZhlnvmLe2EzMzpOLJnm4tk+2nZ24jBsBk6dIZotdmzXWysKHOwbYWUpys7d3dTWVa8Xzd2rtbYhFA6ya08PDqeDi+f7SSSSpd02AInoEsHqOtyGxaUPzlDZvQOfQ26F8htgreAuS/+lCY4+/wnmLp3i5Pkpdj/Ug1mi94+1rr9L5/uJVIbp6m7H5XI+kK66Qk+Bg46uViqrIlw8108qlS7di16ZHHjySez5Id597XXe7Yuye0dD2Wf+4u7ZqUW++e++wVvvnWU+ZuPSGYbPn2Mh7yXgKnZ01yokAjn6Lg7gcjvZsbMTt9v1wNoG02HS0tpIU3M9fRcHiUXjpds2AK17H6XJG+Pdn7/Dz968QM+uLgxpHGSYABSe6hZ+7WtfwFYmTodBNpPBdLoK4+3FDu8GColAhr7LQzQ211NVXfHAx+vWPr+2vgaH00nfxQG6e7vwej0lNlaoScyM8PaZBT75wlMsXjxO1Y59dDRKvYC4e4a3kt/4p18jkTN54tA/ZmFqGjwBDu6sK6npqms9An0XB6msqqC+sXZTrk+lFBVVERxOB0P9I3TsaCMYCpRY2wBWconXfnqSxz/9LJ7EOG+EWjiwu6Hk4iwGSQZYffJ1XUnv3R5PEaO5Na01+Vyegb4hGhrrNiURuJpSisqqCKAZuDzEzl07cLlL69EouTDNzFKS5alRTpyZ5aWvfBa/nOniHihlUtfWvv7/wVBpJpe2ZTPYN0xFZWTTEoE1SimCoQAdO9oYGhilp7cTT4k9LOTiK4xNLNK7Ms+Zd8/w1BdeojFcWu1XscgwwRajtWZ4cJSKygjVNZVFudAKCUEFVTWVDA2MYlnWpsdwc4rqnYf4lU/txzL8fPJXf4We5nBJNUhCPAhaa8ZGJ3C5XZueCKxZSwiamusZ7B/BsqySGjLw1LTyxV/7FOThkec/w5H9LSXVs1NM8ry0hWitmZ9dwLZ10S72NUop6htqiccSzM7MU99Q3HiuZjhc1Le3FzsMITaN1prlpRXisQS9e7qLOiVaKUVVTSXxeIKJsWla25uKFstHKcOksqGJytKeBFIU0jOwRWityWazTE3O0trRXBLrHyilaGtvZnZmnkw6U+xwhChb+bzF2Ogk7Z2tmGbxp/4qpWhqaWRlOUo8nih2OGIDin9HERs2OT5DVU3F+lKgxaaUwuV2UVtbzeT4dEl1BwpRLrTWzM3MEwj48Qd8JdE2ADgcJk0t9YyPTmKv7oUiSpckA1tEJpMluhKjrr6mZC52KCQENXVVxOMJMukSX39AiG0on7eYm1ugsbm+2KFcQylFRWUE27KJx6R3oNRJMlBktm0TjcbI5XLr64B/1FqtQGVVBKez9Fb4Mk2TqupKZmfmix1KUeWyOWKx+HrRlCRG4l5orYlGY2Sz2Vu2DctLK/j9vge2lsC9UEpR11DDzPRcWV8P+XyeWDROPl+6bYMUEBZZPpfnwtk+PF43TqeTSEWIcDiEx+terwuwbZulxWV29HQWOdobWysYunxhgKaWhpIYsyyGWCxO36UhPB43Xp+HSEWYUDi4vhhUqTXUorRprbl0vn91G2EnoXCQcCSEz+9dbxu01szPLdDUXJpz5ZVSRCoiTI5Nk83mcJfYNOTNkk5lOH/2Ml6vG5fbvdrOB3G73SijNNqGkkgG8nmLleWVksyWHrRcLo9t29TWVpPJZJidnmOofwTTNIlUhvF5vfgCPpRh4PGW7lbKLpcTp9PB4sLy6oldfscyuhzD6/VQXVNJMpFkdGicdDqDx+smHAnh9/uorb9/S8KWA9u2WV5aKcsxZ60L+440NNZhWRaLC0uMDo+DUlRUhPH6PIQjIXLZPH6/r9jh3pRpGvgCPhbnl3C5nOgybBuSiRQOh0lNbTWpZIrJsSn6Lw3hcjuJrB7LuvpanE5H0dqGEkkG8kxPzaHL8IK3LItUIsX83CIajWXbWKtjbPF4En/At36Due9/dzZNPJ5Go/AEAnhcd/9Er5QiFA4yOTGNWQIzHYohlUqTzeYwDIVlFc7lXDZHIpEkuhIjHAkRCPoJhgJFjnTr0FozOzNPPpcvdiibTgOpRJqF+SUMQ5HPW9i2JhFPkIwn8fq8rKzE8Pt9GOZ9vOa0JpdJk0xm0MrAFwzgdNzbaqwVFWFGhsdxu11l+dCXzeZIJpIszC9i23Zh8bh8nkQ8SSwaJxgK4PP5Vhd0K46SSAbcbhe79nQXO4yiyGZznD11gWAoQCKewDZNGprqqagIEQwHcThM+i8PEQj67/vfPXPxPb7xjR8wO7fCC1//H/n0Y3e/J7tSimA4QDQaZ+eurvsY5daxuLDM5Pg0Xp+XZCKJy+2iY0cbkYoQ/oAP0zBLpktwqzAMg57e8jyftNaczJwlFA6SSqawLJva+ioikS5CkSBOh4PxsckHMLtIM/Dua3zzW68zt5Lja//iD3m4t/quP00phT/gw+VysXP3jpKYFr3Z4rEEA33D+AN+4vEEDqeT1vZmIhUhAsEApmnctw3m7lZJJAPlPJ5qmub6uHJ1Zws+rxe1umuGUoUnzFw2/0CW9azfe5Q/+F938X98/X8ikb73Jy+3y0Uumyvb4+n2uPB43esrsLlWC7rK8Xdxv5Tz709rTSgcxOl0UN3WdE2tgFIKrTXJZIqq6vvda6joeeI5/vmuA/zr3/+XZHL3vsKo0+XCti20rTHKcIdAp8tZ6OX1eahrqLkmgSuV87skkoFy5nCYdO+8Uhj40RPDti1s297ALALN8mgfr7xxkZYIHD9+AXdtO1/87Zep8uT58LWf8PovzpJ3+HniM5/msYMdmIaJy+W6fscunefkj19hIbSbTzzWSX5lir/79jt87AufoT5w87oFY7Vw0LbtsiwiDAT868eyVC5wsbXt6OkAbnw+re1T4nTdvm249IufcG7RhWdpkNOXp2k99CQv//IT6Pg8P/37f+DMpUk81U18+gsvsqMpgmGauFzO69oGbeV46/vfxdn1GI/saSI2M8ArPzrHc194gYj35jd5pUApoyxrPwA8HnfJtw3ll6KVmLUnn5s9AdmrY8/GBvbYjM+N88o3v8V7g1me/9JLPPFwJ9g2Z1/9O/787z7k8c++yLNPdvGdP/mPnB+LcvOhO5uBD9/jwwtTAOTji/z8x2+xlM7d8u83DAPDNMpyfBe45XEU4k7d9nzShSLD26+tr5k4/yF//ad/TSzYycu/8Tm6m8PY2STf/Y//Fx9MKF789ZfZVZniG//7X7KUunlPgLZtLrzzFpeGFwBILU/xxqvvkLjNAqRKKRxOk1y+lPYx2TxboW2QnoESpylk1RtlBuv47Jeep6u2UGOgc1G+9cpbNO37FEGHRodqqXImOHluhL2t++5rrEqBQmGXYYGQEJtNr/17g+1DTfcBXvzsMXyOwhsSMxd5+91Rjn31U9i5HLXtnSR+8F2G56JUtt3B0MMGL3fDKN+ega1AkoFtxhOpJOy90m2ocxmWVlKs9J/hJ0uDgMZR20591R1ORdrw/V3fU9WxEOLBCNfV4blq9C4XjxJNZDh//E3GnAZa23Ts6cXnvpPbwh0k/vrOHmzE5pJkoMQZhlpfsWojXUwf7YpSbh/1dRU0Pv4cX/nMfpQCbVlgOik8Uhg4HIp8Lo9ev1gNXC4HmXQaDSSjUVIb6Pq3bbtQL+Aov3oBITabUoV/bXSq3kfbD1e4itrqCM9++Tc43BkBwMpbON3u9dcbhip07a81DgpcTgeZTAaNJrm0RHoDPf9aF6aQOxxyyylVW+rIaFuj0WU1NaUw3cTAsqy7+96Gj2deepb/8P/8LX+bnqbGrxgbnuThF15mb2cFGF569rTx4x9+n2rrIDsfOkhLXZDO3d189y9e5fuNNnPn3iWRuX333tr82Y0WD2qtC9XF93OOtChLVyfMpTwue78ZhrG+psWd8kSaeebjPXznG/+J+acOYmaijM3lefkrn6Mq6MTh9dPdVs1r3/0B9uxu9j52iNqgi86d7XzrP/+IHzrm6T/+DrkNJSO68KCwwTZMa122hcjFUtLJwFq17MpKlImxKYb6hnjo4f20d7UVO7RNY5omDodJJp297YyCyvbd/PZXawi4r1xASik6jjzDPwk38MH755ldMOk4cIjWurWFbww+9uXfxnz150yMTtDQvQeUYudTL/DbeT9DMyscev4l9h5L0XiLmQRQWEnSMG69ul4+nye2EmNqYpqBviF29HSya1/vhn8fQkChbbAsa/1c6r88SHdvF717dhY7tE2jlMLrdZNKpvB6Pbd6Jbs//hmqcqFrhvAM08XTv/aPqHv/fc5dGMXwBjn4+H6C3kL7oRxeXvzd3yHy6ttMjEzQcXA/KuThkc9+gYz7dWZXMjz95a/wWDRDhe/WCZhl2YVix1sk/pZlEYvGmZmaZeDyILX1NTz82KE7+I2Ie1EyycDVXV2JeIK52XkG+4YY6h9henKGZCKJUrBjZ1fZrWDl8XpIJlO32Z5U4auq50jV9TuXKcNBc+8+mnuvFAyuf45SuEPVfOKlz13zHoc3wNHPvMjRj77+FhLxJL6AF7j2eCaTKRZmFxgaGGawb4jpyRli0TgoqK6pKrvjCaU7vagUXX1+pJIp5ucWGB4YYaBviOmJGWLRGFpramqrr3v9duf3+4jHk1RURm7ZNtR37+a6lkEpHG4fe48eY+/Rq//4yvz3YE0zz3/5V695jytQySde/tx1r78ZrTWpZAq324Vpmtccn0w6w/zcIqNDo/RfHmRqYpro8gq2rXnq2WNldSzXlOVyxGsHOp/PMzE2yYfvnWJuZp5YNMbS4jLWR6ahaA0fHP+Q8dHxYoRbFFXVVXT2dLAwv0RNbdVdf86tTrAb/0zdUbGP1rqw5G44iGVZTI1Pc/KD00xPzhCPxllcXLp+yqGGMx+eY2V5ZeN/0RYXCod4/GOPFjuMkrfWNliWxczULCeOn2RmaoZYLM7SwjK53PXTXM+dOl9W51IgFGTP/l0sLixvuKboRm75PqWuKwi+m78nGo3hD/iwbZv52QVOvHuSyfEp4rE4iwtLZDPZ695z+UI/5bTHidvj4YmnHsNVpM2cip4MDPUP89YbxxkZGiWxgT2vL567xMVzlzYhutLQ2t7Mnod2k05NYVlWyRbgWJZFPJZAoXn1ldcYvDxILBa/7fuGBoYZGhh+8AGWiIamOh47dkR6BjZgYmySt372DgN9g0RXYrd9/UDfEAN9Q5sQWWmoqqli30N7yOctctkcbk/pbmS2vLiCP+Dju3/zAy5f6GN56fZJ29TEFFMTU5sQXWnw+3088vih8kwGlFK0drRQWVXB0MAI506fZ256npXlFXK5/A27iFrbWwhXhIsQbXHU1FXjdrtwezysLMeoqq4odkjX0VoTi8Zxe9y0tDdTWV3J8MAIZ0+dZ352nqXFZfKruzN+VENTPdW1d7/u+VZTURnh+mctcSP1jXU899lnGRuZ4PSJM8xOz936XGpuoLrm7nvPtppQOIjb7SIcCbK4sER9Y13JJZlaa5KJFChFc0sjVdWV7Nq3kw/fO8X87AIL84vksrkbHs/qmioamhuKEHVxeDzuoj7sFT0ZcDqdVFRVEKmMsP/gXjLZLNHlKMMDIwwNjDA2Mk4iliCXK6x5/8TTj7Nn/65ihr2plFIoQ1HXUMPk+DQVlZENrUa4mdZ2lqurr8bhcBCpCHPg8D72PrSbbDZHbCXGyNAowwMjjA6NEY3G1vcwOPzoQR578kixv8LmUWx4kZhyppTC4XAQjoQJhUP07ukpnEvRGKNDY1fOpZUo2WxhyODwkYfK7lwyDIOa2moG+oaora8pyer7mek5qqsrcTgdhFxBgqEeunfuIJvNkognGR0ZZ7h/mJGhUZaXVshms6Bh175ennvh2WKHv3lWj2exlEyfc2G5SgcOpwO/30d9Yx2PHD1MOpVmcnyK4cFRRgZHcXvcZTmPPRQKMGHbxKKFrXBLhdaaRDxJLpcnFA5eU3zkcDhwOBz4fF5q62s4fOQg6XSGmakZhgdHGR4YxuvzluXxFBt33blUV8OhIw+RSWeYniycS0P9Q3h9HgyzuDu/FYPH68btcbO4sEx1TWVJff90OkMsGqe5tfGatsF0mHgdXrw+L1U1lRw4tJdsJsvs9BwjQ6MM9g8TCPilbdhEJZMMfNRaAxAIBujZ1U137w6svFVSJ/pmUkrR1NLA+NgUgaC/ZJ4AtNaMj03S2FR/y6x2rQHwB3x07GinY0c7lvXEJkYqtgulFKZp4vP76OzuoGNHOx97pnzPJcMwaGquZ6BvhIrKcMnUFWmtmRibora+Gqfz5jGtHU+vz0tbZyutHS0cfepxtJalizfTllntZa3noFwzRaUU4UgIl9PB7Mx8SUy50VozN7uAYRhEKkIbTtTWFoZZe9oT4l5cfS6V68OCz+8jFA4yMTZVEm0DwPLSCulUmtra6js6LoXjaW5gp1ZxP22ZZEAUtHY0Mzs9RyKRLOpFr7UmlUozPTlDa3tz2TbCQpQCpRTNLQ2sLMdYWY4WvW3IZDKMDk/Q1tkiK4xuEXKUthClFC6Xi9b2Zob6R8hmc0W56LXW5HJ5Bi4P0dzaiMfjlmRAiCJzOB107mhjZGiMVCpdtLbBsmwG+0aoq68mEPBL27BFlHUyoO3Ccsf5/PXTGNfWOv/ofxebUopIRZjq2ioGLg/ddArmg6K1Jp/P0395iKrqSiqrKkrqYl+Lb2362dW/mxsd01I5rkLcD/6Aj+bWRvovD5FOZza9bbBtm6GBEbyrRcOl1DbcyNqy1vlcHsuybnofKIf2omwHbLPZHMODI9iWRT5X2ASoZ9cOHKuFLqlkir6L/YUldv0+qmuraGppLImTWylFXX0ttm3Tf2mQru52XG7XA49Na002m2Ogb5hQOEhDic1rtm2bsZEJkvEEttbksjl27u7B4y30XFh5i75LAywvLuN2uwhGQnR1d5TUdxDiXiilqKyqwLY1fattg8/n3ZS2IZ+3GBoYwel00tLWWPIbymmtWVpYYnpqFsMwSCaSNLU0ricxhQLISSbHptBo/AE/HTva8fvvcPv3LaK0j9YDdPHsRZbml9i5u4dd+3aysrzChXOX1jM/j8eD2+PhxLsnaWxuoLa+psgRX8swFI2N9VRVV3LpwgCxaPyBZq1aa+KxBJfO91NZGaGpqR5VQusdaK2ZmZrl/OkL7OjtYve+XvwBP2///Pj6iqamw6Smtprjb75PMBKipU1qHcT2o5SiuqaS5pYGBi4PsbS48sDbhlQqzeUL/Xi9Hlrbm0tmttOt5HN5fvHa29Q31rFzdzfdvV289fo7JOJXVsKtrq1ioG+IlaUVOrs78HputSHU1la2yYA/GKC+sQ4o7AzY0FTPpXOXsazCfgjKUCzMLVBTV01VTSUu14N/8r5TylDU1lfT1tnM8OAo46OT933YYK3bfWJ8mqGBUVrbm6lrqCmpRGCN0+Wkpa0J0yzsnFjfUMvo0BiJRBIoNJIryyt4PG6aW5uk1kFsW0opKiojdHV3MDkxzfDg2AOpMbIsi9npOfouDlDXUENzayPmVikYVNDY0oDX60EpRSBY2Ml1fHSi8GOlsC2b6PIKnT2deL2ebV0MWbbDBHX1tQwNDHPqgzN4vB5s2yYWja8vi6m1ZnhwlM4d7SXd3aWUIhQKsmtvD+Ojk1w4d5mGxjoqqyIYxr0twGJZFkuLK0xNTOPz++jd043TWbrTt0KhIIvzi5z64Awul5NIZYRUMkUmnSEQ9F8177kGl0umLYntTSmFz++ld3c3k+PTXDh7mbqGmvXVAO+FbdusLMeYHJ/C4XSyc3c37k0Yqryf1h4CL567jOkwqa6pIpfNFXZUXbWyXFjhsrbuzqZHbkVllwysjRO9/uovOPzYQZpbm4hF47zy9z/C43Wv3/gT8SRLC0u0PPtkyZ8Ea/Os2ztbSSSSTIxNMT05Q2V1BZWVETxX7XV+s+9y9RNDOpVhaWmZhblFnC4n7V2tBAL+W76/2DKZLD/94c9oamlk38E95POFLsBcLo/HW9jAxbZtxkfG2bGzq6QTPCHul7U5+y1tjdTUVTExNsXs9ByRighV1RV4fZ5rVga8kavbhmwmy/JSlLnZeQzDoLm1kVA4eMv3lyJta059cIb52XmOPv04Xq+HS+f7GBsZZ//hwlbvWmvGRyaoqIwQCASKHPGDV3bJgGVZvPGTN6lvrFsfM/YHfKSSaXbt27k+1rW4sLS6mM6VTZFs2yadzqx3K5WStXj8fh89vV2kkmnmZhfo7xsGXagyDgb9uNwunE7neneXbdnkcrnCuu+xOMl4Ek1h+eOu7g68Ps81n1+KtNacfP8UyUSSvQ/txjRNTNMkm8nS3NqI11co+EkmUizML/H0p5rWv09hI5VCkWgpf0ch7oVSCo/HTVd3O5lMlvnZBYYGR9G2jdfnJRQK4Ha7cbgc622gbdvkc3my2SzxWJJ4PIG2bfwBP+2drfgDvvXP3kq01szOzvHuW+/zxd98eb3A0ufzrvcWwOrDw+gELe3N68OiWmsy6Uxh6fxttmDa9vo2G5BKpRkfneCRo4fXK0YT8STZTJYdPV3rJ/bYyBiV1ZX4VitHtdYsLy6zuLBEV09nMb/CLa3F7/N7aetoxrJsMukMyUSSWCxBdnEZK1+YQqNXX+8wTVwuJ8GQn/r6GtweD4ahtsxFblkWF89eZv+hvesNWS6bY2lhiUOPHkSp1dUSZ+ZwuhxEKiLr781msvRdHODA6tOAENvV2vVcqJlppLG5vtA2JFPEogmWl6LkrTza1mtvwDRNXC4H/oCfmtpK3B4P5jbY/2FkcJRQOEggEFi/D0yOT9HW1UYwVOgFyGZzzM7Mse/gnmve2395kM4d7ZIMbHUKcLmc69NDtNacOXmO3r09VFYVtge2LIvRwTE6ezowDGN1Ra0spz88y4HD+7fUhWCaBj6/F5/fS3Vt1epc2Stdf4Wlgbdedv9RhmkQjhR6cQr1HiP4/L71qYNaayZGJ6mprcbtca3PLz576hwVlaW1VoIQm8EwDLy+1c2CqiuvrMFhrz0oXFk6fPtRBIJBTIe5ugV7jKH+EZ55/qn1Nn95cZlsJktdQy1Q6CmYmZpleXEZzzacVVB2yYDX52P/oX2F+fk9HYyNTIDWPPrkEQzTIB5P0Hehn6XFZXLZPBfOXiKTyTBweRCvx7M+PrZVrd38t9M+uqZp8ugTjzA8OEKkMszS4jLjoxN8/LmncLqcZLM5hvqHGewfJlIR5vKFfqx8nrGRCRbmF3n5y79c7K8gRNGt1w6Y26dtuJne3d3MTc8yMTaJw+Hg4rlLPPrkI9TW1673Epw+cRaP18Po8DhKKZYWlrh47jLP/NLT23JWgdIbnGvyF9/6c+Yn5/jy1768JeaQ3oplWczPLpCIJwiFQ1RURda/Uz6fJxFPYtt2ochMAatP0h6vZ8tVzJYLbWuWl1dYWljC6/NSXVOFY3Xmg23bJOLJwrTRtV4QDba2cTmdUi8gRJnRWpNOpZmdnkMpRVVt1XrtwNrPMpksUJjCvXYPWJuCuBWmT1qWxTf/77/kpS9/jvaG2w9tl13PABSeJNe6fj7K4XAQjoQ2OSJxr5RRmFddURm57meGYayPAwpx9zR2Pk88nkRj4A/6C+PnxQ5L3DGl1PqWyTf7mdfnLUJkxVP66Y0QQhSb1tj5DGeOf8DkXJSl6XHee/cCWWv7rlUvyoskA0IIcRsamBu8yPCySVdXM209XVizA1wcWdzWm9eI8lFWwwTru07dy7W7rStstyat9ZXpUHdJbaGplKIYNJMDQzj8uzCVQuEk7FX0DUyxv6Oy2MGJOyD3gRsru2Tg/JlLTI5P3fVnOJ0ODj96cMvPKthOxkbGuXSu766vbQXsObCLxuaG+xmW2FY0KytxzJCDtfbf4TKJzkUL0/CKGpu4E8lEkvfe+ZDsaoHg3ahrqGHfQ3skGdiqlFJ07+yko+v6opE7+JT15W1FaWhY3b3xXrjcckzFLWiwVzcxW6MorOApthavz8uRxw/d0/CO6TC3VSIAZZgMuD1u3EjDv504XU6csvGQeJCUwut1k7Js1roCLMvG7fVIr8AWYxjG+sqy4oqyLyDUWpPL5sjn7+/Wv6J41rZdzuXu/5atolwpahrryCWT2Gi0tognczQ01RQ7MHEf2LZNNpNd37W2HJVVz8BH5XI5hgdGsCyLbCaHz++ls7tDdrTbwgo7E06sb6pia03v7h7pORD3RAHNu/bQ99pZ5ldS+O0oi3k/D++UOpOtTGvN0uIyU+NTOJwOEvEkHTvaiVSEt90wwO2U9V3v7MnzxGMJdu7uYc+BXYyPTDDYPyxPk1uU1pqpiWn6Lw2wc1c3u/fvAuC9d07IMRX3RincoVqOHdvHyuQ449MxDj/9BNUBZ9ndNLaTTCbLW6+/TUt7Cz27umnvauPnP32TTDpT7NA2XVknA06ng0hFGK0L40gNzQ2cP31BbhxbmYLK6sr18u6mlkb6Lw6U5cUt7i+lFIGqWnp299C7u5vaCr8kAttAOBLG6Sx0kkcqwmRSGaYmZ4oc1eYr22ECrTWd3R2MDI1x6dwlPF4PuWyWlaUVbMtmdmaORCxBa0crTqeDRDxBNpsry+6jraS2toZ0KsOlc5dxulz4/F4SiQTpdIZUKs387DxNLY34/D6yq9sc19bXyDEVogy5XE56du1goG8Ih8NBVU0luXyO6HKUTDrDxNgkgWCAmrpqAGan56isqtiWw45l2zOwuLDEf/7BT/D5fXT2dBAIBTh14ixen5e52XliKzFOvn+a6YkptNaceO8Ul85fLnbY4ia01qTTGX76o9dJxpN0dndQ11DL2ZPnsW1NLpdjYnSSmalZzpw8B8Do0Chv/uztIkcuhCgG27Y5/cEZzp46T1tHCx1dbUxPTDM1MYPX76X/8iBaa37yw5+htSaVSvOj779KMpkqdugPRFkmA7lcnp/9+A0amhtpbW/G7XYTDodIxBN07mgnl8tTW19DLBrDF/BjWRaTY1PU1skTZCk78e5JUskUu/f14vF6CIYCJJNJWtuaAGhua2RmepZIRQStNcODozQ01Rc5aiFEMcxMzfLeOyd4+LHDBIIBXG4XHq8Hp9NJdU0V4UiIaDRGKBxEoViYW8Q0Tbw+T7FDfyDKMhlIxOJMjk/R1tGy/mfxeALbsunY0U5LWxNzM/N4fV4iFWFSyTSxaIzam+x0KIrPyltcOHORrp5OTEdhO+pMJsvKcpTefb1U11ShbU1sOUZzayO5XJ65mXkaW6QaXIhyo7Vm4PIglVUVBIJXaj+mJqbpXJ1NUNdQx1DfMN29O0DB5Pgk1bVVOJ3bb4gAyjQZ0FrjdF3Zx962bc6ePMeeA7sJr9YETE3M0NTSiGEYTE1ME46E8XhksaJSVVhrXBNZ3cJYa83wwAjhcIj2jlaUUiwuLBGKhPB4PCRicZKJJNW11dLbI0QZsiybUCS0PpU8Fo0xNjzOkScexjAM8vk80ZUYdQ21aFszPDBKe2frtm0vyrKAMBgOsXN3D0P9QzS1NDExNgHAkaOH1w90Q1Md/ZcGGR+Z4P23P6ClvRnTNIsZtrgFh9PBI0cfZqh/GK/Pw9LCMmPD43zil57GsVYpXBnBNE0mxiY5f+YCgWAAr3d7dvkJIW5tz/5dvPPzd5manAYNF89d5tEnj6wXCzocDuoaapkcm1ztSZyjpm77LjJVlsmAaRo8+fHHmZmaJboSpa6hjj0Hdq/f7PP5PD6/j0NHHiKfz5PP52nvatu2GeF2oJRi38E9zM0Uij99fi9PPfskTldhHrjWmnwuz+NPPYpl2WQyWXbs7JRjKkQZUkpRVVPJx559goX5JQzD4JHHDxEIBtbbi1QqxcFHDoAqDB9U11Zv6w3qyjIZUErhdDppbm264c+XFpZ54ydv8uLLz9N3cYDWjlYpNNsCDMOg7iZ1HZlMYabBM88/TSadxjAMdu/rlWRAiDKllCIUDhEKh677mdaaE8c/JBQO0dXTyfjIBE8+/TgOx/a9ZW7fb3YPwpEQew7sYrBvCH/Ax579u7b1SVAOXC4Xhx99iJmpWQyleOaXnsYjQwRCiBtQSrFrXy8zk7MMD4zw2JNHqK6tKnZYD5Tc4W7A5Xax98ButNYopeTpcRswDIPu3h1yTIUQt6WUorauhpqrCoy3e5shycBNyA1j+5FjKoTYqHJrL8pyaqEQQgghrpBkQAghhChzkgwIIYQQZU6SASGEEKLMSTIghBBClDlJBoQQQogyJ8mAEEIIUeYkGRBCCCHKnCQDQgghRJmTZEAIIYQocxtfjljB0sIS77/9PoYhOYQQQghRqmzbZmVpBTa4ovKGkwGvz8uTzzxJOp1B29bdxieEEEKIB0wDx549hsezsd1ZN5wMOB0ODh45eLdxCSGEEKJEKa21LnYQQgghhCgeGfwXQgghypwkA0IIIUSZk2RACCGEKHOSDAghhBBlTpIBIYQQosxJMiCEEEKUOUkGhBBCiDInyYAQQghR5iQZEEIIIcrc/w+gkj6Fj7QIpwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mask = (x <= 0)\n",
    "print(mask)\n",
    "# 如果正向传播时的输入值小于等于0,则反向传播的值为0;反向传播中会使用正向传播时保存的mask,将从上游传来的dout的mask中的元素为True的地方设为0\n",
    "# 正向传播时,有电流通过的话,就将开关设为ON;没有电流通过的话,就将开关设为OFF\n",
    "# 反向传播时,开关为ON的话,电流就会直接通过;开关为OFF的话,则不会有电流通过\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.image import imread\n",
    "img = imread('./image/ch05_1.png')\n",
    "plt.imshow(img)\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3fce80b2-4d77-42e9-aba9-9e863176761a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 激活函数的Sigmoid层\n",
    "class Sigmoid:\n",
    "    def __init__(self):\n",
    "        self.out = None\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = 1 / (1 + np.exp(-x))\n",
    "        self.out = out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        dx = dout * (1.0 - self.out) * self.out\n",
    "\n",
    "        return dx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6aeda0b7-2842-4aca-a5f9-a38d8d9976c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Affine层:神经网络的正向传播中进行的矩阵的乘积运算在几何学领域被称为\"仿射变换\",Affine层就是仿射变换的实现\n",
    "# 几何中,仿射变换包括一次线性变换和一次平移,分别对应神经网络的加权和运算与偏置运算\n",
    "class Affine:\n",
    "    def __init__(self, W, b):\n",
    "        self.W = W\n",
    "        self.b = b\n",
    "        self.x = None\n",
    "        self.dW = None\n",
    "        self.db = None\n",
    "\n",
    "    def forward(self, x):\n",
    "        self.x = x\n",
    "        out = np.dot(x, self.W) + self.b\n",
    "\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        dx = np.dot(dout, self.W.T)\n",
    "        self.dW = np.dot(self.x.T, dout)\n",
    "        self.db = np.sum(dout, axis=0)\n",
    "\n",
    "        return dx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "27cc1465-1b4c-4818-a992-d685073fd331",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Softmax-with-Loss层:Softmax层 + Cross_Entropy_Error层\n",
    "# Softmax:将输入值正规化之后再输出(一般不用)\n",
    "# 实现Softmax-with-Loss层\n",
    "class SoftmaxWithLoss:\n",
    "    def __init__(self):\n",
    "        self.loss = None # 损失\n",
    "        self.y = None # Softmax输出\n",
    "        self.t = None # 监督数据(one-hot vector)\n",
    "\n",
    "    def forward(self, x, t):\n",
    "        self.t = t\n",
    "        self.y = softmax(x)\n",
    "        self.loss = cross_entropy_error(self.y, self.t)\n",
    "\n",
    "        return self.loss\n",
    "\n",
    "    def backward(self, dout=1):\n",
    "        batch_size = self.t.shape[0]\n",
    "        dx = (self.y - self.t) / batch_size\n",
    "\n",
    "        return dx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "740bf44a-ed04-47a7-bde2-612c95811005",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 误差反向传播法的实现\n",
    "# params 保存神经网络的参数的字典型变量.params['W1']是第一层权重,params['b1']是第一层的偏置\n",
    "# layers 保存神经网络的层的有序字典型变量\n",
    "# lastLayer 神经网络的最后一层\n",
    "\n",
    "# __init__(self,input_size,hidden_size,output_size,weight_init_std) \n",
    "#         初始化,参数从头开始依次是输入层的神经元数、隐藏层的神经元数、输出层的神经元数、初始化权重时的高斯分布的规模\n",
    "# predict(self, x) 进行识别(推理).参数x是图像数据\n",
    "# loss(self, x, t) 计算损失函数的值.参数x是图像数据、t是正确解标签\n",
    "# accuracy(self, x, t) 计算识别精度\n",
    "# numerical_gradient(self, x, t) 通过数值微分计算关于权重参数的梯度\n",
    "# gradient(self, x, t) 通过误差反向传播法计算关于权重参数的梯度\n",
    "\n",
    "# TwoLayerNet代码实现\n",
    "import sys, os\n",
    "sys.path.append(os.pardir)\n",
    "import numpy as np\n",
    "from common.layers import *\n",
    "from common.gradient import numerical_gradient\n",
    "from collections import OrderedDict\n",
    "# OrderedDict是有序字典,\"有序\"是指它可以记住字典里添加元素的顺序\n",
    "# 因此神经网络的正向传播只需要按照添加元素的顺序调用各层的forward()方法就可以完成处理,反向传播只需要按照相反的顺序调用各层即可\n",
    "\n",
    "class TwoLayerNet:\n",
    "\n",
    "    def __init__(self, input_size, hidden_size, output_size, weight_init_std=0.01):\n",
    "        # \n",
    "        # 初始化权重\n",
    "        self.params = {}\n",
    "        self.params['W1'] = weight_init_std * \\\n",
    "                            np.random.randn(input_size, hidden_size)\n",
    "        self.params['b1'] = np.zeros(hidden_size)\n",
    "        self.params['W2'] = weigh_init_std * \\\n",
    "                            np.random.randn(hidden_size, output_size)\n",
    "        self.params['b2'] = np.zeros(output_size)\n",
    "\n",
    "        # 生成层\n",
    "        self.layers = OrdereDict()\n",
    "        self.layers['Affine1'] = \\\n",
    "            Affine(self.params['W1'], self.params['b1'])\n",
    "        self.layers['Relu1'] = Relu()\n",
    "        self.layers['Affine2'] = \\\n",
    "            Affine(self.params['W2'], self.params['b2'])\n",
    "\n",
    "        self.lastLayer = SoftmaxWithLoss()\n",
    "\n",
    "    # 数据流动:输入->Affine1->ReLU->Affine2->输出\n",
    "    def predict(self, x):\n",
    "        for layer in self.layers.values():\n",
    "            x = layer.forward(x) # 依次通过各层\n",
    "        return x # 输出层未经过Softmax的原始得分\n",
    "\n",
    "    # x:输入数据, t:监督数据\n",
    "    def loss(self, x, t):\n",
    "        y = self.predict(x) # 前向传播\n",
    "        return self.lastLayer.forward(y, t) # 计算Softmax + 交叉熵\n",
    "\n",
    "    def accuracy(self, x, t):\n",
    "        y = self.predict(x)\n",
    "        y = np.argmax(y, axis=1) # 预测类别\n",
    "        if t.ndim != 1 :\n",
    "            t = np.argmax(t, axis=1) # 将one-hot转为索引\n",
    "        accuracy = np.sum(y==t) / float(x.shape[0]) # 正确率\n",
    "        return accuracy\n",
    "\n",
    "    # x:输入数据, t:监督数据\n",
    "    def numerical_gradient(self, x, t):\n",
    "        loss_W = lambda W: self.loss(x, t) # 损失函数闭包\n",
    "\n",
    "        grads = {}\n",
    "        grads['W1'] = numerical_gradient(loss_W, self.params['W1'])\n",
    "        grads['b1'] = numerical_gradient(loss_W, self.params['b1'])\n",
    "        grads['W2'] = numerical_gradient(loss_W, self.params['W2'])\n",
    "        grads['b2'] = numerical_gradient(loss_W, self.params['b2'])\n",
    "\n",
    "        return grads\n",
    "\n",
    "    def gradient(self, x, t):\n",
    "        # forward(前向传播,保留中间结果)\n",
    "        self.loss(x, t)\n",
    "\n",
    "        # backward(反向传播)\n",
    "        dout = 1\n",
    "        dout = self.lastLayer.backward(dout) # 从输出层开始\n",
    "\n",
    "        layers = list(self.layers.value())\n",
    "        layers.reverse()\n",
    "        # layers = list(self.layers.values())[::-1] # 同,反转层顺序\n",
    "        for layer in layers:\n",
    "            dout = layer.backward(dout) # 逐层反转传播\n",
    "\n",
    "        # 设定 收集梯度\n",
    "        grads = {}\n",
    "        grads['W1'] = self.layers['Affine1'].dW\n",
    "        grads['b1'] = self.layers['Affine1'].db\n",
    "        grads['W2'] = self.layers['Affine2'].dW\n",
    "        grads['b2'] = self.layers['Affine2'].db\n",
    "\n",
    "        return grads\n",
    "        # SoftmaxWithLoss->Affine2->ReLU->Affine1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "813c66bb-529a-48e8-a778-15b925bf0f97",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W1:4.965029934213904e-10\n",
      "b1:2.8346496756944883e-09\n",
      "W2:5.461212430645957e-09\n",
      "b2:1.403028204505219e-07\n"
     ]
    }
   ],
   "source": [
    "# 梯度确认:在确认误差反向传播法的实现是否正确时,是需要用到数值微分的\n",
    "# 梯度确认代码实现\n",
    "import sys, os\n",
    "sys.path.append(os.pardir)\n",
    "import numpy as np\n",
    "from dataset.mnist import load_mnist\n",
    "from ch05.two_layer_net import TwoLayerNet\n",
    "\n",
    "# 读入数据\n",
    "(x_train, t_train), (x_test, t_test) = \\\n",
    "    load_mnist(normalize=True, one_hot_label=True)\n",
    "\n",
    "network = TwoLayerNet(input_size=784, hidden_size=50, output_size=10)\n",
    "\n",
    "x_batch = x_train[:3]\n",
    "t_batch = t_train[:3]\n",
    "\n",
    "grad_numerical = network.numerical_gradient(x_batch, t_batch)\n",
    "grad_backprop = network.gradient(x_batch, t_batch)\n",
    "\n",
    "# 求各个权重的绝对误差的平均值\n",
    "for key in grad_numerical.keys():\n",
    "    diff = np.average(np.abs(grad_backprop[key] - grad_numerical[key]))\n",
    "    print(key + \":\" + str(diff))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb331ead-b49c-4f62-959e-8652a2dda05f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.09133333333333334 0.0897\n",
      "0.9043166666666667 0.9072\n",
      "0.9200166666666667 0.9211\n",
      "0.93385 0.9317\n",
      "0.94085 0.9394\n",
      "0.9487666666666666 0.9469\n",
      "0.9536166666666667 0.9506\n",
      "0.9575166666666667 0.9567\n",
      "0.9605333333333334 0.9579\n"
     ]
    }
   ],
   "source": [
    "# 使用误差反向传播法的学习(和之前的实现相比,不同之处仅在于通过误差反向传播法求梯度这一点)\n",
    "import sys, os\n",
    "sys.path.append(os.pardir)\n",
    "import numpy as np\n",
    "from dataset.mnist import load_mnist\n",
    "from ch05.two_layer_net import TwoLayerNet\n",
    "\n",
    "# 读入数据\n",
    "(x_train, t_train), (x_test, t_test) = \\\n",
    "    load_mnist(normalize=True, one_hot_label=True)\n",
    "\n",
    "network = TwoLayerNet(input_size=784, hidden_size=50, output_size=10)\n",
    "\n",
    "iters_num = 10000\n",
    "train_size = x_train.shape[0]\n",
    "batch_size = 100\n",
    "learning_rate = 0.1\n",
    "train_loss_list = []\n",
    "train_acc_list = []\n",
    "test_acc_list = []\n",
    "\n",
    "iter_per_epoch = max(train_size / batch_size, 1)\n",
    "\n",
    "for i in range(iters_num):\n",
    "    batch_mask = np.random.choice(train_size, batch_size)\n",
    "    x_batch = x_train[batch_mask]\n",
    "    t_batch = t_train[batch_mask]\n",
    "\n",
    "    # 通过误差反向传播法求梯度\n",
    "    grad = network.gradient(x_batch, t_batch)\n",
    "\n",
    "    # 更新\n",
    "    for key in ('W1', 'b1', 'W2', 'b2'):\n",
    "        network.params[key] -= learning_rate * grad[key]\n",
    "\n",
    "    loss = network.loss(x_batch, t_batch)\n",
    "    train_loss_list.append(loss)\n",
    "\n",
    "    if i % iter_per_epoch == 0:\n",
    "        train_acc = network.accuracy(x_train, t_train)\n",
    "        test_acc = network.accuracy(x_test, t_test)\n",
    "        train_acc_list.append(train_acc)\n",
    "        test_acc_list.append(test_acc)\n",
    "        print(train_acc, test_acc)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
